/**
 * This file is part of Sonedyan.
 * 
 * Sonedyan is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License as published by the Free Software Foundation;
 * either version 3 of the License, or (at your option) any
 * later version.
 *
 * Sonedyan is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied
 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 * PURPOSE.  See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public
 * License along with Octave; see the file COPYING.  If not
 * see <http://www.gnu.org/licenses/>.
 * 
 * Copyright (C) 2009-2012 Jimmy Dubuisson <jimmy.dubuisson@gmail.com>
 */

package org.unige.mpej.eckmann.sonedyan.enron.db;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Set;
import java.util.Vector;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.mail.Address;
import javax.mail.internet.InternetAddress;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.unige.mpej.eckmann.sonedyan.enron.bean.Actor;
import org.unige.mpej.eckmann.sonedyan.enron.bean.Email;
import org.unige.mpej.eckmann.sonedyan.enron.bean.EnronMessage;

/**
 * class used to load the Enron dataset
 */
public class DatasetLoader 
{
	private static Logger log = Logger.getLogger(org.unige.mpej.eckmann.sonedyan.enron.db.DatasetLoader.class);
	
	private static final String DEFAULT_DATASET_BASE_FOLDER = "./maildir";
	private static final String LIST_DEFAULT_FILENAME = "list.txt";
	private static final String ACTORS_DEFAULT_FILENAME = "actors.csv";
	
	private static final String[] NOT_ENRON_PRO_DOMAINS = { "aep.com", "ey.com", "avistaenergy.com", "iso-net.com", "ubswenergy.com" };
	private static final String[] ENRON3_SUBDOMAINS = { "ei", "ect", "ees" };
	
	private DatabaseHandler conn;
	private File baseDirectory;
	
	/**
	 * class constructor
	 */
	public DatasetLoader(String baseFolderPath, DatabaseHandler conn)
	{
		this.conn = conn;
		this.baseDirectory = new File(baseFolderPath);
	}
	
	/**
	 * class constructor
	 */
	public DatasetLoader(DatabaseHandler conn)
	{
		this.conn = conn;
		this.baseDirectory = new File(DEFAULT_DATASET_BASE_FOLDER);
	}
	
	/**
	 * write the set of messages (517'582) in the form 'fileAbsolutePath'
	 */
	private boolean writeMessageList()
	{
		PrintWriter out = null;
		
		try
		{
			log.info("Getting list of messages in folder: " + this.baseDirectory.getAbsolutePath());
			
			File listFile = new File(LIST_DEFAULT_FILENAME);
			
			if (!listFile.exists())
			{
				out = new PrintWriter(new BufferedWriter(new FileWriter(listFile)));
	
				if (this.validateBaseFolder(baseDirectory))
				{
					Iterator<File> iterator = FileUtils.iterateFiles(baseDirectory, null, true);
					
					while (iterator.hasNext())
					{
						File file = iterator.next();
						
						if (file.isFile())
						{
							out.println(file.getAbsolutePath());
						}
					}

					return true;
				}
				else
				{
					log.error("Unable to get list of messages: the specified base directory is invalid");
					return false;
				}
			}
			else
			{
				log.info("The file containing the list of messages already exist");
				return true;
			}
		}
		catch(Exception e)
		{
			log.error("An error occured: " + e.getMessage());
			return false;
		}
		finally
		{
			if (out != null)
			{
				try
				{
					out.close();
				}
				catch(Exception e){}
			}
		}
	} 
	
	/**
	 * load dataset
	 */
	public boolean loadDataset()
	{
		BufferedReader br = null;
		
		try
		{
			if (writeMessageList())
			{
				log.info("Loading dataset in the repository");
				
				br = new BufferedReader(new FileReader(LIST_DEFAULT_FILENAME));
				
				String line;
				
				while ((line = br.readLine()) != null)
				{
					EnronMessage message = new EnronMessage(this.baseDirectory.getAbsolutePath(), new File(line).getAbsolutePath());
					
					log.info("Adding message '" + message.getId() + "' with path '" + message.getRelativeFilePath() + "' to the repository");
					
					// store the message
					this.conn.storeMessage(message);
				}
				
				return true;
			}
			else
			{
				return false;
			}
		}
		catch(Exception e)
		{
			log.error("Unable to load dataset: " + e.getMessage());
			return false;
		}
		finally
		{
			if (br != null)
			{
				try
				{
					br.close();
				}
				catch(Exception e){}
			}
		}
	}
	
	/**
	  * Directory is valid if it exists, does not represent a file, and can be read.
	  */
	private boolean validateBaseFolder(File baseDirectory)
	{
		if (baseDirectory == null)
		{
			log.error("Base directory should not be null");
			return false;
		}
		else if (!baseDirectory.exists())
		{
			log.error("Base directory does not exist");
			return false;
		}
		else if (!baseDirectory.isDirectory())
		{
			log.error("Base directory should be directory");
			return false;
		}
		else if (!baseDirectory.canRead())
		{
			log.error("Base directory cannot be read");
			return false;
		}
		
		return true;
	}
	
	/**
	 * check if the specified email is an enron1 email
	 * 
	 * NB: the parameters are supposed not to be null
	 */
	private boolean isEnron1Email(String email, String firstName, String lastName)
	{
		if (email.endsWith("@enron.com") && (email.startsWith(firstName + ".") || email.startsWith(firstName + "_")) && email.contains(lastName + "@"))
		{
			return true;
		}
		else
		{
			return false;
		}
	}
	
	/**
	 * check if the specified email is an enron1 email
	 * 
	 * NB: the email string is supposed not to be null
	 */
	private boolean isEnron3Email(String email)
	{
		for (int i = 0; i < ENRON3_SUBDOMAINS.length; i++)
		{
			if (email.endsWith("@" + ENRON3_SUBDOMAINS[i] + ".enron.com"))
					return true;
		}
		
		return false;
	}
	
	/**
	 * check if the specified email is an enron1 email
	 * 
	 * NB: the email string is supposed not to be null
	 */
	private boolean isNotEnronProEmail(String email)
	{
		for (int i = 0; i < NOT_ENRON_PRO_DOMAINS.length; i++)
		{
			if (email.endsWith("@" + NOT_ENRON_PRO_DOMAINS[i]))
				return true;
		}
		
		return false;
	}
	
	/**
	 * get an actor instance from the specified parameters
	 * 
	 * NB: for known actors, actor's ids take the form 'fitrstname.lastname'
	 */
	private Actor getActor(String[] columns)
	{
		String firstName = columns[0].substring(1, columns[0].length() - 1).toLowerCase();
		String lastName = columns[1].substring(1, columns[1].length() - 1).toLowerCase();
		String sex = columns[2].substring(1, columns[2].length() - 1);
		String position = columns[3].substring(1, columns[3].length() - 1);
		String emails = columns[4].substring(1, columns[4].length() - 1).toLowerCase();
		
		// generate an id for the new actor of the form 'firstname-lastname'
		String id = firstName + "." + lastName;
		
		log.info("Adding actor " + firstName + " " + lastName);
		
		Actor actor = new Actor();
		actor.setId(id);
		actor.setFirstName(firstName);
		actor.setLastName(lastName);
		
		if (sex.equals("MALE")){ actor.setSex(Actor.Sex.MALE); }
		else { actor.setSex(Actor.Sex.FEMALE); }
		
		if (position.equals("CEO")){ actor.setHierarchyLevel(Actor.HierarchyLevel.CEO); }
		else if (position.equals("PRESIDENT")){ actor.setHierarchyLevel(Actor.HierarchyLevel.PRESIDENT); }
		else if (position.equals("VICE_PRESIDENT")){ actor.setHierarchyLevel(Actor.HierarchyLevel.VICE_PRESIDENT); }
		else if (position.equals("MANAGING_DIRECTOR")){ actor.setHierarchyLevel(Actor.HierarchyLevel.MANAGING_DIRECTOR); }
		else if (position.equals("DIRECTOR")){ actor.setHierarchyLevel(Actor.HierarchyLevel.DIRECTOR); }
		else if (position.equals("MANAGER")){ actor.setHierarchyLevel(Actor.HierarchyLevel.MANAGER); }
		else if (position.equals("LAWYER")){ actor.setHierarchyLevel(Actor.HierarchyLevel.LAWYER); }
		else if (position.equals("TRADER")){ actor.setHierarchyLevel(Actor.HierarchyLevel.TRADER); }
		else if (position.equals("EMPLOYEE")){ actor.setHierarchyLevel(Actor.HierarchyLevel.EMPLOYEE); }
		else if (position.equals("UNKNOWN")){ actor.setHierarchyLevel(Actor.HierarchyLevel.UNKNOWN); }
		
		actor.setEmails(this.getActorEmails(actor, emails));
		actor.setKnown(true);
		
		return actor;
	}
	
	/**
	 * get actors emails
	 */
	private Vector<Email> getActorEmails(Actor actor, String emails)
	{
		try
		{
			Vector<Email> v = new Vector<Email>();
			
			String[] emailsArray = emails.split(",");
			
			for (int i = 0; i < emailsArray.length; i++)
			{
				String emailString = emailsArray[i];
				Email email = new Email();
				
				email.setAddress(new InternetAddress(emailString));
				
				if (Email.isEnronEmail(emailString))
				{
					if (this.isEnron3Email(emailString))
					{
						email.setType(Email.Type.ENRON3);
					}
					else
					{
						if (this.isEnron1Email(emailString, actor.getFirstName(), actor.getLastName()))
						{
							email.setType(Email.Type.ENRON1);
						}
						else
						{
							email.setType(Email.Type.ENRON2);
						}
					}
				}
				else
				{
					if (this.isNotEnronProEmail(emailString)){ email.setType(Email.Type.NOT_ENRON_PRO); }
					else{ email.setType(Email.Type.PRIVATE); }
				}
				
				v.add(email);
			}
			
			return v;
		}
		catch(Exception e)
		{
			log.error("An error occured: " + e.getMessage());
			return null;
		}
	}
	
	/**
	 * load the Enron dataset known actors
	 * 
	 * NB: the file expected format is CSV with ';' as a column separator and '"' as text marker
	 * with the following columns: "firstName";"lastName";"SEX";"POSITION","EMAILS"
	 */
	public boolean loadKnownActors()
	{
		BufferedReader br = null;
		
		try
		{
			br = new BufferedReader(new FileReader(ACTORS_DEFAULT_FILENAME));
			
			String line;
			
			while ((line = br.readLine()) != null)
			{
				String[] columns = line.split(";");
				
				Actor actor = this.getActor(columns);
				
				this.conn.storeActor(actor);
			}
			
			return true;
		}
		catch(Exception e)
		{
			log.error("An error occured: " + e.getMessage());
			return false;
		}
		finally
		{
			try
			{
				if (br != null)
				{ 
					br.close(); 
				}
			}
			catch(Exception e){}
		}
	}
	
	/**
	 * load the Enron dataset unknown actors
	 * 
	 * http://www.ietf.org/rfc/rfc2822.txt
	 * 
	 * Automatically extracts unknown emails of the form:
	 * 
	 * john.smith@enron.com
	 * john_smith@enron.com
	 * john.f.smith@enron.com
	 * 
	 * And ignores unknow emails of the form:
	 * 
	 * jsmith@enron.com
	 * jsmith@ect.enron.com
	 * smith@enron.com
	 * f..smith@enron.com
	 * smith36@enron.com
	 * jsmith@hotmail.com
	 * jsmith@ubswenergy.com
	 * 
	 * Here is a list of 'strange' emails:
	 * 
	 * 'lesly'@enron.com
	 * 'mcconnell@enron.com
	 * /.../.../...-...@enron.com
	 * recipients@enron.com
	 * host@mailman.enron.com
	 * 'enron'@enron.com
	 * dad@enron.com
	 * a@enron.com
	 * iii'.'leo@enron.com
	 * iii.a.@enron.com
	 * ship'.'nolan's@enron.com
	 * `caresteam@enron.com
	 * robert.fenton-stone@enron.com
	 * 
	 */
	public boolean loadUnknownActors()
	{
		try
		{
			log.info("Filtering the list of Enron messages");
			
			// get the set of all email addresses
			Hashtable<Integer, Address> emails = this.conn.getAllEmails(); 
			
			log.info("First analysis pass");
			
			Pattern pattern1 = Pattern.compile("([a-zA-Z0-9-]+)\\.([a-zA-Z0-9-]+)@[a-zA-Z\\.]?enron.com");
			Pattern pattern2 = Pattern.compile("([a-zA-Z0-9-]+)\\_([a-zA-Z0-9-]+)@[a-zA-Z\\.]?enron.com");
			Pattern pattern3 = Pattern.compile("([a-zA-Z0-9-]+)\\.[a-zA-Z0-9]+\\.([a-zA-Z0-9-]+)@[a-zA-Z\\.]?enron.com");
			
			// 'middle..last'
			// Pattern badPattern1 = Pattern.compile("([a-zA-Z0-9]+)\\.\\.([a-zA-Z0-9-]+)@[a-zA-Z\\.]?enron.com");
			
			// set of all the email keys
			Set<Integer> keySet = emails.keySet();
			
			// multimap lastName -> actors
			Hashtable<Integer, Actor> actors = new Hashtable<Integer, Actor>();
			// email to check in a 2nd step
			Vector<Email> emailsToCheck = new Vector<Email>();
			
			for (Integer key : keySet)
			{
				Address address = emails.get(key);
				
				// is it an enron email?
				if (Email.isEnronEmail(address))
				{
					// is it a known email?
					Actor actor = this.conn.getActorByAddress(address.toString());
					
					if (actor == null || !actor.isKnown())
					{
						String emailString = address.toString();
						
						log.info("First pass for email: '" + emailString + "'");
						
						String firstName = null;
						String lastName = null;
						
						Matcher matcher = null;
						
						// 'first.middle.last'
						if ((matcher = pattern3.matcher(emailString)).matches())
						{
							firstName = matcher.group(1);
							lastName = 	matcher.group(2);
						}
						// 'first_last'
						else if ((matcher = pattern2.matcher(emailString)).matches())
						{
							firstName = matcher.group(1);
							lastName = 	matcher.group(2);
						}
						// 'first.last'
						else if ((matcher = pattern1.matcher(emailString)).matches())
						{
							firstName = matcher.group(1);
							lastName = 	matcher.group(2);
						}
						else
						{
							log.info("Uncomplete format found (i.e. 'iii.a.@enron.com', 'sandraf.@enron.com')");
							log.info("Adding email '" + emailString + "' to the list of emails to be checked...");
							
							Email newEmail = new Email();
							newEmail.setAddress(address);
								
							if (this.isEnron3Email(emailString))
							{
								newEmail.setType(Email.Type.ENRON3);
							}
							else
							{
								newEmail.setType(Email.Type.ENRON2);
							}
							
							emailsToCheck.add(newEmail);
						}
						
						// add the found last name to the list 
						if (firstName != null && lastName != null)
						{
							log.debug("A complete email of the form 'firstname.lastname@enron.com' was found");
							
							Email newEmail = new Email();
							newEmail.setAddress(address);
							newEmail.setType(Email.Type.ENRON1);
							
							String id = firstName + "." + lastName;
							Actor storedActor = actors.get(id.hashCode());
							Actor newActor = null;
							Vector<Email> actorEmails = new Vector<Email>();
								
							if (storedActor != null)
							{
								log.debug("Actor '" + firstName + "' '" + lastName + "' was found");
								log.debug("Adding '" + newEmail.getAddress().toString() + "' to the set of existing emails");
								
								newActor = storedActor;
								actorEmails = newActor.getEmails();
								actorEmails.add(newEmail);
							}
							else
							{
								log.debug("Adding actor '" + firstName + "' '" + lastName + "'");
								
								newActor = new Actor();
								newActor.setFirstName(firstName);
								newActor.setLastName(lastName);
								newActor.setKnown(false);
								
								log.debug("Adding '" + newEmail.getAddress().toString() + "' to the set actor's emails");
								
								actorEmails.add(newEmail);
								newActor.setEmails(actorEmails);
								
								// id will have the form: 'firstname.lastname'
								newActor.setId(id);
							}
							
							actors.put(id.hashCode(), newActor);
						}
					} // if the email is not known
				} // if is enron email
			} // loop over email set
			
			log.info("Second analysis pass");
			
			// loop over the emails to be checked ('middle..lastname', 'lastname', '<first letter of first name>.lastname', ...)
			//
			// NB: jsmith could be identified to John Smith 
			for (Email email : emailsToCheck)
			{
				String emailString = email.getAddress().toString();
				
				log.info("Second pass for email: '" + emailString + "'");
				
				// add a new actor for each email to be checked
				Actor newActor = new Actor();
				newActor.setId(emailString);
				newActor.setKnown(false);
				
				Vector<Email> actorEmails = new Vector<Email>();
				actorEmails.add(email);
				newActor.setEmails(actorEmails);
				
				actors.put(emailString.hashCode(), newActor);
			}
			
			log.info("Adding the set of unknow actors to the database");
			
			Set<Integer> actorsKeySet = actors.keySet();
			
			for (Integer i : actorsKeySet)
			{
				Actor actor = actors.get(i);
				
				log.info("Storing actor (id: '" + actor.getId() + "', first name: '" + actor.getFirstName() + "', last name: '" + actor.getLastName() + "')");
				this.conn.storeActor(actor);
			}
			
			return true;
		}
		catch(Exception e)
		{
			log.error("An error occured: " + e.getMessage());
			e.printStackTrace();
			return false;
		}
	}
	
}
